# -*- coding: utf-8 -*-

#
# dict_xintong={}
# with open('dict_xintong.txt', 'r', encoding="utf-8") as f:  #839
#     for line in f:
#         dict=line.split('\n')[0]
#         dict_xintong.setdefault(dict)
#
#
# for key in dict_xintong:
#     f1 = open('dict_xintong_1.txt', 'a', encoding="utf-8")
#     f1.write(key)
#     f1.write('\n')
#     f1.close()


# print(len(dict_xintong))
#
# #juzi='又称模拟的基带信号为模拟基带信号'
# juzi='脉冲干扰是幅度随机变化、占空比很小且随机变化的脉冲序列，'
# for key in dict_xintong:
#     if key in juzi:
#         print(key)

# ##按标点分行
# list_biaodian=['。','！','？','，','：','；']

# 语料合并
# 去掉特殊字符，只保留中英文字符
import re
import ner_predict as ner_predict
import datetime
import os
import csv

def clearSpecialSymbols(lines):
    result = []
    for line in lines:
        valid1 = re.search('^[a-zA-Z]',line)
        valid2 = re.search('^[\u4e00-\u9fa5]',line)
        if valid1 or valid2:
            result.append(line)
    result_1="".join(result)
    return result_1

# result_list = re.split(pattern, txt)
# print(result_list)
pattern = r'。|，|；|·|！|？|：'

#book_list=['电子电路基础','现代通信技术','数据结构与STL']

filename = '/mnt/data/kg/NER/PPT/通信原理三级目录/'    # 输入待处理文件的文件地址
outfilename = '/mnt/data/kg/NER/PPTresult/通信原理三级目录/'    # 新文件的地址
pathDir = os.listdir(filename)

for allDir in pathDir:
    txt = open(filename + allDir, 'rt', encoding='utf-8')  # 读取所需要分析的文件内容
    print(txt)
    #allDir = allDir[9:11]   # 用list列表切片方法修改文件名
    allDir = allDir[0:7]
    #allDir = allDir[:-8]
    Excel = open(outfilename + allDir + 'NER.csv', 'w', newline='',encoding='utf-8')  # 打开表格文件，若表格文件不存在则创建
    writ = csv.writer(Excel)  # 创建一个csv的writer对象用于写每一行内容

#for book in book_list:
    #open_file='book\\'+book+'.txt'
    #save_file=book+'词典.txt'
    dict1={}
    #print(datetime.datetime.now(),'打开',book,'')
    #with open(filename, 'r', encoding="utf-8-seg") as f:
    for line in txt:
        if line != '\n':
            line = line.replace(' ', '').replace('\n', '')
            result_list = re.split(pattern, line)
            for item in result_list:
                if item != '':
                    NER_1 = ner_predict.predict_online(item)  # 命名实体识别
                    NER_1 = NER_1.replace("##", '').replace("[UNK]", '').strip(" ")
                    NER_1 = NER_1.replace(' ', '').split(',')
                    for ner_1 in NER_1:
                        if ner_1 != '':
                            dict1.setdefault(ner_1)
                            if dict1[ner_1] != None:
                                dict1[ner_1] = dict1[ner_1] + 1
                            else:
                                dict1[ner_1] = 1
    dict_sort_list = sorted(dict1.items(), key=lambda x: x[1])
    dict_sort_list.reverse()
    #dict_sort=dict(dict_sort_list)
    #print(datetime.datetime.now(),"课本《",book,"》词典的长度：",len(dict_sort))
    #for key in dict_sort:
        #if len(key)>1:
            #f1 = open(save_file, 'a', encoding="utf-8")
            #f1.write(key)
            #f1.write(' ')
            #f1.write(str(dict_sort[key]))
            #f1.write('\n')
            #f1.close()

    for i in range(len(dict_sort_list)):
        if len(dict_sort_list[i][0]) > 1:  # 去掉单字符的词
            writ.writerow(dict_sort_list[i])  # 写入表格



